# coding=utf-8
import time
import traceback

import requests
#from BeautifulSoup import BeautifulSoup version 3.0.0
from bs4 import BeautifulSoup
from pymongo import MongoClient

import Utils

mgClient = MongoClient('localhost', 27017,
    username='root2',
    password='Dream2015',
    authSource="stockDataStore",
    authMechanism='SCRAM-SHA-1')
db = mgClient['stockDataStore']

def get_stock_info(code):
    url = 'http://vip.stock.finance.sina.com.cn/corp/go.php/vCI_CorpInfo/stockid/' + code + '.phtml'
    response = requests.request("get", url);
    response_content = response.content;
    response_content = response_content.decode("GBK").encode("UTF-8")
    soup = BeautifulSoup(response_content, 'html.parser')
    tags = soup.select(".tagmain td")
    title = soup.title.text;
    index_left_parentheses = title.index("(")
    index_right_parentheses = title.index(")")
    name = title[0:index_left_parentheses]
    stock_code = title[index_left_parentheses + 1:index_right_parentheses]
    if not name or not stock_code:
        return None
    # 双汇发展(000895)公司资料_新浪财经_新浪网
    # ()公司资料_新浪财经_新浪网
    stock_info = {}
    stock_info['name'] = name
    stock_info['first_letter'] = Utils.get_first_letter(name)
    stock_info['code'] = stock_code
    for i in range(0, len(tags) - 2, 2):
        name = tags[i].text
        name = name[0:len(name) - 1]  # 去掉最后的':'号
        stock_info[name] = tags[i + 1].text
    return stock_info


stockCodes = []

for i in range(0, 3000):
    stockCodes.append(str(i).zfill(6))

for i in range(0, 800):
    stockCodes.append("30" + str(i).zfill(4))

for i in range(0, 4000):
    stockCodes.append("60" + str(i).zfill(4))

for stockCode in stockCodes:
    try:
        if Utils.bypass_stock(stockCode):
            Utils.logger.info(stockCode + " bypassed ")
            continue

        Utils.logger.info("try to get stock info" + stockCode)
        record = db['stock'].find_one({'code': stockCode})
        if record is not None:
            Utils.logger.info("stock of " + stockCode + " already exist in mongo")
            continue
        time.sleep(3.5)
        stock_info = get_stock_info(stockCode)
        if not stock_info:
            Utils.logger.warn("can not find stock info from sina of " + stockCode)
            continue
        stock_info['code'] = stockCode
        stock_info['first_letter'] = Utils.get_first_letter(stock_info['name'])
        if not db['stock'].find_one({"code": stockCode}):
            db['stock'].insert_one(stock_info)
            Utils.logger.info("success to get stock info" + stockCode)
        else:
            Utils.logger.info("already exits:" + stockCode)

    except Exception as e:
        traceback.print_exc()

Utils.logger.info("CollectStockInfo.py exit")
